#!/usr/bin/env python
"""
Copyright (C) 2009 pk

This work is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2.

This work is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
version 2 of the GNU General Public License for more details.
"""

import os, sys
import Queue
import xmlrpclib
import threading
import MySQLdb
from optparse import OptionParser
import multiprocessing
# from 'portage-server' import BasePortage
from SimpleXMLRPCServer import SimpleXMLRPCServer
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
import time
import logging, logging.handlers
import datetime
import socket
import kthread
import platform

LOG_FILENAME = "logs/portage-master.log"
LOG_FORMAT = '%(asctime)s %(levelname)s %(message)s'

_master_port = 1590
_server_port = 1591
_webserver_port = 8080
_ip = socket.gethostbyname(socket.gethostname())
# socket.setdefaulttimeout(-1)

parser = OptionParser()
logger = logging.getLogger("portage-master")


def add_options():
    parser.add_option("-i", "--ip", dest="ip", default = _ip,
                      help="Ip to bind to")
    parser.add_option("-p", "--port",
                      dest="port", default= _master_port,
                      help="Port to bind to")
    parser.add_option("-n", "--name",
                      dest="name",
                      help = "Server name. Defaults to host hostname")


add_options()


def setup_logger():

    logger.setLevel(logging.DEBUG)
    fh = logging.handlers.TimedRotatingFileHandler(
        "logs/portage-master.log",
        when = 'midnight', backupCount = 7)
    fh.setLevel(logging.ERROR)
    # create console handler with a higher log level
    ch = logging.StreamHandler(sys.stdout,)
    ch.setLevel(logging.DEBUG)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)

setup_logger()

class HelperDefs(object):
    @staticmethod
    def _create_dir(dirname, mode):
        try:
            os.mkdir(dirname, mode)
            return True
        except IOError, (code, msg):
            logger.critical("Could not create dir.\n\t\
            Error code: %d\n\tError message: %s" %(code, msg))
            return False

    @staticmethod
    def _get_bin_path():
        dirs = os.path.abspath(__file__).split('/')
        dirs.pop()
        temp = [cur_dir + os.sep for cur_dir in dirs]
        del dirs
        return os.path.abspath(os.sep + ''.join(temp))


class PortageServer(object):
    _server_port = 1591

    def __init__(self, ip, name = None, server_type = None):
        self.ip = ip
        self.server_type = type
        self.name = name
        self.conn = xmlrpclib.ServerProxy('http://' +\
                                          self.ip + ':' +\
                                          str(self._server_port))



class SpawnProcess(multiprocessing.Process):
    """Portage Package command in process
    Queue contains arguments"""
    def __init__(self, in_queue, out_queue, func):
        multiprocessing.Process.__init__(self)
        self.in_queue = in_queue
        self.out_queue = out_queue
        self.func = func

    def run(self):
        for args in iter(self.in_queue.get, None):
            output = self.func(*args)
            self.out_queue.put({args[0]: output})        


class BasePortage(SimpleXMLRPCServer):
    """
    BasePortage sublclass of SimpleXMLRPCServer.
    Creates an rpc server on specified port and ip and registers
    functions defined in class ExportFunctions via xml-rpc
    """
    # Default portage-master hostname
    _portage_master = "portage-master"

    # Server name defaults to node name(short hostname)
    name = platform.node()


    class ExportFunctions(object):
        """
        Define functions to be exported via xml-rpc
        """
        
        # Package status
        _status = {
            'R' : 'Installed',
            'N' : 'Not installed',
            'U' : 'Update available'
            }

        def __init__(self, server):
            self.server = server


    def connect_master(self, ip, port):
        self.master = xmlrpclib.ServerProxy('http://' + str(ip) + ':' + str(port), verbose=False)
        self.master.add_server(self.name, self.ip, self.type)
        

    def __init__(self, ip, port, type, name = None, alone = False, master = None):
        self.ip = ip
        self.port = port
        if name:
            self.name = name
        self.type = type
        if master:
            self.portage_master = master
        else:
            self.portage_master = self._portage_master
        
        class RequestHandler(SimpleXMLRPCRequestHandler):
            rpc_paths = ('/RPC2',)
        SimpleXMLRPCServer.__init__(self, (self.ip, self.port),
                                    requestHandler=RequestHandler, allow_none = True)
        self.register_multicall_functions()
        self.register_introspection_functions()
        self.register_instance(self.ExportFunctions(self))
        if not alone:
            self.connect_master(self.portage_master, _master_port)
        self.service = kthread.KThread(target = self.serve, args = [])
        # self.service.daemon = True
        self.service.start()

        
    def serve(self):
        try:
            self.serve_forever()
        except KeyboardInterrupt:
            logger.info("Exiting..")
            sys.exit(0)



class PortageMaster():
    _db_name = 'portage_master'
    _db_user = 'portage-master'
    _db_pass = 'portage-master'
    _db_host = 'localhost'
    _TYPE_FILES_DIR_NAME = 'type_files'
    _THREAD_NO = 5
    _FILE_DIRS = {'etc' : ['make.conf'], # 'hosts'],
                  'portage' : ['package.use', 'package.mask', 'package.unmask', 'package.keywords'],
                  'make.profile' : []}

    _FILES = {'make.conf' : 'etc',
              'profile' : 'etc',
              # 'hosts' : 'etc',
              'package.use' : 'portage',
              'package.unmask' : 'portage',
              'package.keywords' : 'portage',
              'package.mask' : 'portage'}


    def __init__(self, ip, port):
        self.ip = ip
        self.port = port
        self.export_functions = PortageMaster.ExportFunctions(self)
        self.db_conn = self._init_db(self._db_name, self._db_user, self._db_pass, self._db_host)
        self.binpath = HelperDefs._get_bin_path()
        self._types_dirpath = os.path.abspath(self.binpath + os.sep + self._TYPE_FILES_DIR_NAME)
        self._make_type_dirs()
        self.server = PortageService(self.ip, self.port, self.export_functions)

    def __del__(self):
        self.db_conn.close()


    def _init_db(self, _db_name, _db_user, _db_pass, _db_host):
        try:
            db_conn = MySQLdb.connect(host = _db_host,
                                      user = _db_user,
                                      passwd = _db_pass,
                                      db = _db_name)
        except MySQLdb.Error, e:
            logger.critical("Error %s" % (e))
            return
        return db_conn


    def _set_paths(self):
        pass

    def add_server(self, name, ip, type_name):
        logger.info("%s - %s connected, checking if in database" %(ip, name))
        server_id = self.get_server_id(ip)
        if server_id:
            logger.info("Server with ip %s already exists in database" %(ip))
            return None
        type_id = self.get_type_id(type_name)
        if type_id:
            self._run_statement("""
            INSERT INTO Portage_Server(server_name, server_ip, type_id)
            VALUES ('%s', INET_ATON('%s'), %d);
            """ %(name, ip, type_id))
            try:
                self.db_conn.commit()
            except MySQLdb.Error, e:
                logger.critical("Error %d: %s" %(e.args[0]. e.args[1]))
                sys.exit(1)
                
        else:
            msg = "Type %s does not exist" %type_name
            logger.error(msg)
            return msg
        return "Here be dragons.."
    
    def add_type(self, server_type):
        type_id = self.get_type_id(server_type)
        if type_id:
            return "Error - Type %s already exists!" % (server_type)
        statement = """
        INSERT INTO Server_Type(type)
        VALUES ('%s');
        """ %(server_type)
        output = self._run_statement(statement)
        try:
            self.db_conn.commit()
        except MySQLdb.Error, exception:
            logger.critical("Error %d: %s" %(exception.args[0]. exception.args[1]))
            sys.exit(1)
        return output


    def db_set_last_seen(self, id, datetime, db_conn):
        """
        Sets last_seen value to that of datetime for the id
        given
        """
        statement = """
        UPDATE Portage_Server
        SET last_seen = '%s'
        WHERE server_id = %d;
        """ % (datetime, id)
        self._run_statement(statement, db_conn)
        self._db_commit(db_conn = db_conn)

    def db_get_last_seen(self, id, db_conn = None):
        """
        Gets last_seen value for server_id
        """
        statement = """
        SELECT last_seen
        FROM Portage_Server
        WHERE server_id = %d;
        """ % (id)
        rows = self._run_statement(statement, db_conn = db_conn)
        if rows:
            return rows[0][0]
        logger.error("No last_seen value for server_id %d" % (id))
        return None


    def db_increment_failure(self, id, period, db_conn = None):
        """
        Increments failure period in DB for id
        """
        statement = """
        UPDATE Portage_Server
        SET failure_period = failure_period + %d
        WHERE server_id = %d;
        """ % (period, id)
        self._run_statement(statement, db_conn = db_conn)
        self._db_commit(db_conn = db_conn)


    def db_set_connectable(self, id, connectable, db_conn = None):
        """
        Sets connectable value for server_id
        """
        statement = """
        UPDATE Portage_Server
        SET connectable = '%s'
        WHERE server_id = %d;
        """ % (id, connectable)
        self._run_statement(statement, db_conn = db_conn)
        self._db_commit(db_conn = db_conn)
            

    def _db_commit(self, db_conn = None):
        """
        Attempts a db commit on given db_conn, or global one
        """
        if not db_conn:
            db_conn = self.db_conn
        try:
            db_conn.commit()
        except MySQLdb.Error, exception:
            logger.critical("Error %d: %s" %(exception.args[0]. exception.args[1]))
            return False
        return True


    def get_types(self):
        statement = """
        SELECT type.type
        FROM Server_Type type;
        """
        rows = self._run_statement(statement)
        try:
            types = [type for (type,) in rows]
        except TypeError, e:
            logger.info("No servers in type %s" %(type))
            return None
        return types


    def get_server_id(self, ip, db_conn = None):
        """
        Get server_id from given IP
        """
        if not ip:
            return None
        statement = """
        SELECT server.server_id
        FROM Portage_Server server
        WHERE server.server_ip = INET_ATON('%s');
        """ %(ip)
        server_id = self._run_statement(statement,
                                        db_conn = db_conn)
        if server_id:
            return server_id[0][0]
        return

                                   
    def get_type_id(self, type_name, db_conn = None):
        statement = """
        SELECT type.type_id
        FROM Server_Type type
        WHERE type.type = '%s';
        """ %(type_name)
        server_id = self._run_statement(statement,
                                        db_conn = db_conn)
        if server_id:
            return server_id[0][0]
        return None


    def get_type(self, type_id):
        if not type_id:
            return None
        statement = """
        SELECT type.type
        FROM Server_Type type
        WHERE type.type_id = %s;
        """ %(type_id)

        server_type = self._run_statement(statement)
        if server_type:
            return server_type[0][0]
        return None


    def get_server_ips_by_type(self, server_type, db_conn = None):
        """
        Gets server ips by server type, ignoring
        servers where failure period is at max
        """
        type_id = self.get_type_id(server_type,
                                   db_conn = db_conn)
        statement = """
        SELECT INET_NTOA(server.server_ip)
        FROM Portage_Server server
        WHERE server.type_id = %d
        AND failure_period != 255;
        """ %(type_id)
        rows = self._run_statement(statement, db_conn = db_conn)
        try:
            ips =  [ip for (ip,) in rows]
        except TypeError:
            logger.info("No servers in type %s" %(server_type))
            return None
        return ips

    def get_server_ip_by_id(self, server_id):
        """
        Gets server ip for server_id
        """
        statement = """
        SELECT INET_NTOA(server.server_ip)
        FROM Portage_Server server
        WHERE server.server_id = %d;
        """ % (server_id)
        rows = self._run_statement(statement)
        if rows:
            return rows[0][0]
        logger.info("No server with server_id %d" % (server_id))


    def get_server_ids_by_type(self, server_type, db_conn = None):
        """
        Returns server ids that correnspond to server type
        """
        type_id = self.get_type_id(server_type,
                                   db_conn = db_conn)
        statement = """
        SELECT server.server_id
        FROM Portage_Server server
        WHERE server.type_id = %d;
        """ %(type_id)
        rows = self._run_statement(statement, db_conn = db_conn)
        try:
            ids =  [id for (id,) in rows]
        except TypeError:
            logger.info("No servers in type %s" %(server_type))
            return None
        return ids
        

    # def 

    def get_type_id_by_server(self, ip):
        if not ip:
            return None
        statement = """
        SELECT server.type_id
        FROM Portage_Server server
        WHERE server.server_ip = INET_ATON('%s');
        """ %(ip)

        server_type = self._run_statement(statement)
        if server_type:
            return server_type[0][0]
        return None
            

    def get_packages_by_type(self, server_type):
        if not server_type:
            return None
        type_id = self.get_type_id(server_type)
        
        statement = """
        SELECT package.package_name
        FROM Package package
        INNER JOIN Server_Type_to_Package type2package
        ON package.package_id = type2package.package_id
        WHERE type2package.type_id = %d;
        """ %type_id
        
        rows = self._run_statement(statement)
        try:
            packages =  [package for (package,) in rows]
        except TypeError:
            logger.info("No servers in type %s" %(server_type))
            return None
        return packages

    def _create_dirs_tree(self, prefix, dirs, mode):
        try:
            current_dir = dirs.pop()
        except IndexError:
            return None
        dirpath = os.path.abspath(prefix + os.sep + current_dir)
        if not (os.path.exists(dirpath)):
            success = HelperDefs._create_dir(dirpath, mode)
            if not success:
                logger.critical("Could not create dir %s" %(self._types_dirpath))
                success = False
                return None
            else:
                prefix = dirpath
                logger.info("Created dir %s" %(self._types_dirpath))
                return self._create_dirs(prefix, dirs)
            
    def _create_dirs(self, prefix, dirs, mode):
        try:
            dir = dirs[len(dirs)-1]
        except IndexError:
            return None
        dirpath = os.path.abspath(prefix + os.sep + dir)
        if not os.path.exists(dirpath):
            success = HelperDefs._create_dir(dirpath, mode)
            if not success:
                logger.critical("Could not create dir %s" %(dirpath))
                success = False
            else:
                logger.info("Created dir %s" %(dirpath))
                # Sending copy of dirs so as not to change the original
                # reference to the array
        self._create_dirs(prefix, dirs[:len(dirs)-1], mode)

    def _make_type_dirs(self):
        mode = 0750
        type_dir_names = self.get_types()
        final_success = True
        
        if not os.path.exists(self._types_dirpath):
            if not HelperDefs._create_dir(self._types_dirpath, mode):
                final_success = False

        prefix = self._types_dirpath
        if not self._create_dirs(prefix, type_dir_names, mode):
            final_success = False

        for type_dir in type_dir_names:
            prefix = os.path.abspath(self._types_dirpath + os.sep + type_dir)
            # logger.debug("Prefix: %s, dirs: %s" %(prefix, self._FILE_DIRS.keys()))
            if not self._create_dirs(prefix, self._FILE_DIRS.keys(), mode):
                final_success = False
        return final_success

    def _run_statement(self, statement, db_conn = None):
        if not db_conn:
            db_conn = self.db_conn
        cursor = db_conn.cursor()
        try:
            cursor.execute(statement)
        except MySQLdb.Error, (code, msg):
            logger.error("Statement executed: %s\
            \n\tError code: %d\n\tError message: %s" %(statement, code, msg))
            return None
        if cursor.rowcount:
            rows = cursor.fetchall()
            cursor.close()            
            return rows
        else:
            cursor.close()
            return None
        
    def get_server_ips(self, type):
        statement = """
        SELECT INET_NTOA(server.server_ip)
        FROM Portage_Server server;
        """
        rows = self._run_statement(statement)
        try:
            ips =  [ip for (ip,) in rows]
        except TypeError:
            logger.info("No servers in type %s" %(type))
            return None
        return ips


    def server_stop(self, ip):
        """
        Stops given server
        """
        s = PortageServer(ip)
        try:
            output = s.conn.service_stop()
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output
        

    def portage_install(self, ip, package):
        """ Function that merges (installs)
        given package on given ip
        """
        s = PortageServer(ip)
        try:
            output = s.conn.portage_install(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def portage_ebuild_package(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.ebuild_pkg(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def portage_latest_version(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.get_latest_version(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output
            

    def portage_pretend(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_pretend(package)
        except socket.error, e:
            if isinstance(e, tuple):
                (code, msg) = e
                message = "%s for ip %s - Status code - %s" %(msg, ip, code)
                logger.error(message)
                output = "%s - Status code - %s" %(msg, code)
            else:
                message = "%s for ip %s" %(e, ip)
                logger.error(message)
                output = message
        logger.info("%s : %s" % (ip, output))
        return output


    def portage_sync(self, ip):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_sync()
        except socket.error, e:
            if isinstance(e, tuple):
                (code, msg) = e
                message = "%s for ip %s - Status code - %s" %(msg, ip, code)
                logger.error(message)
                output = "%s - Status code - %s" %(msg, code)
            else:
                message = "%s for ip %s" %(e, ip)
                logger.error(message)
                output = message
        logger.info("%s : %s" % (ip, output))
        return output


    def portage_search(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_pretend(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def portage_update_system(self, ip):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_update_system()
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def portage_update_pretend(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_update_pretend(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def portage_update(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_update(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def portage_remove(self, ip, package):
        s = PortageServer(ip)
        try:
            output = s.conn.portage_remove(package)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg, code)
        return output


    def push_make_conf(self, ip):
        type = self.get_type(self.get_type_id_by_server(ip))
        if not type:
            logger.info("There is no type assigned to this ip. %s" %(ip))
            return None
        type_dir = self._FILES['make.conf']
        file_dir = os.sep.join([self._types_dirpath, type, type_dir, 'make.conf'])
        logger.debug("Filepath for make.conf push: %s" %(file_dir))
        # file_dir = self._types_dirpath + os.sep + type + os.sep + type_dir + os.sep + 'make.conf'
        try:
            file_stream = open(file_dir)
        except IOError, (code, msg):
            logger.error("Error reading make.conf file\
            \n\tError code: %d\n\tError message: %s" %(code, msg))
            return False
        bin = xmlrpclib.Binary(file_stream.read())
        server = PortageServer(ip)
        try:
            output = server.conn.push_make_conf_file(bin.data)
        except socket.error, (code, msg):
            message = "%s for ip %s - Status code - %s" %(msg, ip, code)
            logger.error(message)
            output = "%s - Status code - %s" %(msg,code)
        return output


    def push_make_conf_type(self, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        logger.info(server_ips)
        return self.run_command_threaded(self.push_make_conf, server_ips)


    def portage_push_files_ip(self, ip, server_type, file_to_do = None):
        """
        Pushes files to IP according to server type
        """
        db_conn = self._init_db(self._db_name, self._db_user, self._db_pass, self._db_host)
        # server_ips = self.get_server_ips_by_type(server_type, db_conn = db_conn)
        data = {}
        _FILES = self._FILES
        if file_to_do:
            if not file_to_do in self._FILES:
                logger.error("File given %s is not in allowed files list %s"
                             % (file_to_do, self._FILES))
                return
            _FILES = [file_to_do]
                             
        for filename in _FILES:
            type_dir = self._FILES[filename]
            filepath = os.sep.join([self._types_dirpath, server_type, type_dir, filename])
            try:
                file_stream = open(filepath)
            except IOError, (code, msg):
                logger.error("Error on %s at %s.\n\tCode: %d\
                Message: %s" %(filename, filepath, code, msg))
            bin = xmlrpclib.Binary(file_stream.read())
            data[filename] = bin.data
        logger.debug("Pushing data to server: %s" %(ip))
        server = PortageServer(ip)
        output = self._try_xml_rpc(server.conn.portage_push_files, data)
        if isinstance(output, tuple):
            logger.debug("Error pushing files to %s" % (ip))
            self._check_error_code(output[0], ip, server_type,
                                   db_conn = db_conn)
            db_conn.close()
            return output[1]
        else:
            db_conn.close()
            return output


    def _check_error_code(self, code, ip, server_type, db_conn = None):
        """
        Checks error code and performs actions necessary
        """
        codes = { 128 : self.__error_code_128,
                  146 : self.__error_code_146,
                  145:  self.__error_code_128
                  }
        if code in codes:
            codes[code](ip, db_conn = db_conn)
        else:
            logger.error("Got unknown error code %d, please implement" % (code))



    def __error_code_128(self, ip, db_conn = None):
        """
        Handling of Network unreachable error,
        increment and check failure period
        Also used for connection time outs
        """
        if not db_conn:
            db_conn = self.db_conn
        now = datetime.datetime.now()
        server_id = self.get_server_id(ip, db_conn = db_conn)
        if not server_id:
            return
        self.db_set_connectable(server_id, 0, db_conn = db_conn)
        last_seen_datetime = self.db_get_last_seen(server_id, db_conn = db_conn)
        if last_seen_datetime:
            self.db_increment_failure(server_id, int((now-last_seen_datetime).seconds/60),
                                      db_conn = db_conn)
        else:
            self.db_increment_failure(server_id, 50, db_conn = db_conn)


    def __error_code_146(self, ip, db_conn = None):
        """
        Handling of connection refused error,
        """
        if not db_conn:
            db_conn = self.db_conn
        server_id = self.get_server_id(ip, db_conn = db_conn)
        if not server_id:
            return
        self.db_set_connectable(server_id, 0,
                                db_conn = db_conn)

    
    def _try_xml_rpc(self, func, *args):
        try:
            output = func(*args)
        except socket.error, e:
            if isinstance(e, tuple):
                (code, msg) = e
                # if code == 111 and msg == 'Connection refused':
                #                 logger.fatal("Portage master (%s:%d) is not listening."%(self._portage_master,
                #                                                                          self._portage_master_port))
                #                 sys.exit(1)
                logger.fatal(" - Error code: %d - Error Status: %s" %(code, msg))
                return code, msg
            else:
                logger.fatal(" - Error: %s" % (e))
                return str(e)
        except socket.gaierror, (code, msg):
            if code == -2:
                logger.fatal("Unable to lookup hostname.%s - %s" %(code, msg))
            return code, msg
        return output


    def _process_worker(self, in_queue, out_queue):
        """
        Bug in multiprocessing does not allow Process to have a target
        within the instance of the object it is being run from. This
        function therefor does not work with multiprocessing.Process.
        The subclass SpawnProcess is used as a workaround
        """
        for func, args in iter(in_queue.get, None):
            output = self.func(*args)
            out_queue.put({args[0]: output})

            
    def _thread_worker(self, in_queue, out_queue):
        """ Helper def to iterate through input queue
        and execute function with arguments as given
        by each member of the queue
        
        Used as a target def of threading.Thread
        """
        for func, args in iter(in_queue.get, None):
            output = func(*args)
            out_queue.put({args[0]: output})
            in_queue.task_done()
            out_queue.task_done()

    
    def run_command_processes(self, func, server_ips_package):
        """
        Runs a ThreadPortagePackageCommand object
        which runs the given function where the function
        takes (ip, package) as its paramaters
        
        Parameters are function name
        and list containing ips and package name as the last
        member of the list. .pop() on the list will give you
        package name, rest are ips to run command on
        """

        package = server_ips_package.pop()
        ips = server_ips_package
        logger.info("Running on ips: %s - Package: %s" %(ips, package))
        start = time.time()
        queue = multiprocessing.Queue()
        out_queue = multiprocessing.Queue()
        output = []

        # Start worker processes
        for _ in range(4):
            proc = SpawnProcess(queue, out_queue, func)
            proc.start()
            # multiprocessing.Process(target=self._process_worker, args=(queue, out_queue)).start()
        for ip in ips:
            # queue.put((func, (ip, package)))
            queue.put((ip, package))

        for _ in range(4):
            queue.put(None)

        for _ in range(len(ips)):
            output.append(out_queue.get())
            # Got result. Push none in queue to end
            # process that has returned output

        elapsed =  "Elapsed Time: %s" % (time.time() - start)
        output.append(elapsed)
        return output
    

    def run_command_threaded(self, func, args):
        """
        Runs a threading.Thread
        which runs the given function where the function
        takes (ip, package) as its paramaters
        
        Parameters are function name
        and list containing ips and package name as the last
        member of the list. .pop() on the list will give you
        package name, rest are ips to run command on
        """
        
        start = time.time()
        queue = Queue.Queue()
        out_queue = Queue.Queue()
        output = {}

        # Spawn as many threads as we have servers to run commands on
        # Operating system will take care of rest and if we're running a long
        # command we can queue them all up and await feedback.
        for _ in range(len(args)):
            t = threading.Thread(target=self._thread_worker, args=(queue, out_queue))
            t.setDaemon(True)
            t.start()

        # put data to be run by threads in queue
        # If single argument we need to cast it to tuple on the list
        for arg in args:
            if (type(arg) == tuple):
                print arg
                logger.debug("Argument is tuple: %s" %(arg,))
                queue.put((func, (arg)))
            else:
                logger.debug("Argument is list: %s" %(arg))
                queue.put((func, (arg,)))

        # Push none for each thread to kill threads that have
        # finished processing
        for _ in range(len(args)):
            queue.put(None)
            
        for _ in range(len(args)):
            thread_output = out_queue.get()
            for key in thread_output:
                output[key] = thread_output[key]

        output['elapsed-time'] = "%s" % (time.time() - start)
        return output

    def servers_stop_type(self, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips = [(ip) for ip in server_ips]
        logger.info("Stopping servers: %s" % (server_ips))
        return self.run_command_threaded(self.server_stop, server_ips)

    
    def portage_install_type_package_threaded(self, package, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_install, server_ips_package)

    
    def portage_pretend_type_package_threaded(self, package, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_pretend, server_ips_package)


    def portage_update_pretend_type_package(self, package, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_update_pretend, server_ips_package)

    
    def portage_update_type_package_threaded(self, package, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_update, server_ips_package)


    def portage_remove_type_package(self, package, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_remove, server_ips_package)


    def portage_sync_type(self, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips = [(ip) for ip in server_ips]
        logger.info(server_ips)
        return self.run_command_threaded(self.portage_sync, server_ips)


    def portage_pretend_type_world_threaded(self, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, 'world') for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_pretend, server_ips_package)


    def portage_pretend_type_package_process(self, package, server_type):
        server_ips_package = self.get_server_ips_by_type(server_type)
        server_ips_package.append(package)
        logger.info(server_ips_package)
        output = self.run_command_processes(self.portage_pretend, server_ips_package)
        return output

        
    def portage_search_type_package_threaded(self, package, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_package = [(ip, package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_search, server_ips_package)

        
#     def portage_pretend_process(self, server_ips_package):
#         return self.run_command_processes(self.portage_pretend, server_ips_package)

#     def portage_ebuild_package_threaded(self, server_ips_package):
#         return self.run_command_threaded(self.portage_ebuild_package, server_ips_package)
    
    def portage_ebuild_package_process(self, server_ips_package):
        return self.run_command_processes(self.portage_ebuild_package, server_ips_package)
    
    def portage_latest_version_threaded(self, server_ips_package):
        # server_ips = self.get_server_ips_by_type(type)
        # server_ips_package = [(ip,package) for ip in server_ips]
        logger.info(server_ips_package)
        return self.run_command_threaded(self.portage_latest_version, server_ips_package)
        # return self.run_command_threaded(self.portage_latest_version, server_ips_package)

    def portage_latest_version_process(self, server_ips_package):
        return self.run_command_processes(self.portage_latest_version, server_ips_package)


    def portage_update_type_system_threaded(self, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        logger.info(server_ips)
        return self.run_command_threaded(self.portage_update_system, server_ips)


    
#     def push_portage_files_type(self, type):
#         server_ips = self.get_server_ips_by_type()
#         logger.debug(server_ips)
#         pass

    def portage_push_files_type_threaded(self, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_type = [(ip, server_type) for ip in server_ips]
        logger.info("Pushing files to %s" %(server_ips))
        return self.run_command_threaded(self.portage_push_files_ip, server_ips_type)

    def portage_push_file_type_threaded(self, file, server_type):
        server_ips = self.get_server_ips_by_type(server_type)
        server_ips_type = [(file, ip, server_type) for ip in server_ips]
        logger.info("Pushing file %s to %s" %(file, server_ips))
        return self.run_command_threaded(self.portage_push_file_ip, server_ips_type)
    
    def portage_update_type_system_process(self, server_ips):
        output = self.run_command_processes(self.portage_update_system, server_ips)
        logger.debug(output)
        return output

    def service_stop(self):
        threading.Thread(target = self._exit, args = []).start()
        return True

    def _exit(self):
        time.sleep(5)
        self.server.service.kill()
        sys.exit(0)

#     def portage_ebuild_package_threaded(self, package, type):
#         server_ips = self.get_server_ips_by_type(type)
#         server_ips_package = [(ip,package) for ip in server_ips]
#         logger.info(server_ips_package)
#         return self.run_command_threaded(self.portage_ebuild_package, server_ips_package)





    class ExportFunctions(object):
        """
        Functions in this class will be exported
        via XML-RPC
        """
        def __init__(self, master):
            self.master = master
            self.output = Queue.Queue()

        def service_stop(self):
            return self.master.service_stop()

        def servers_stop_type(self, server_type):
            return self.master.servers_stop_type(server_type)

        def add_server(self, name, ip, type_name):
            self.master.add_server(name, ip, type_name)

## Needs rewriting to use DB
#         def set_type(self, ip, type):
#             self.master.set_type(ip, type)


        def portage_push_files_ip(self, ip, server_type):
            return self.master.portage_push_files_ip(ip, server_type)
        
        def portage_pretend(self, package, ip):
            return self.master.portage_pretend(ip, package)

        def portage_install(self, ip):
            return self.master.portage_install(ip)

        def portage_update_pretend_type_package(self, package, server_type):
            return self.master.portage_update_pretend_type_package(package, server_type)
        
        def portage_update_type_package(self, package, server_type):
            return self.master.portage_update_type_package_threaded(package, server_type)

        def portage_remove_type_package(self, package, server_type):
            return self.master.portage_remove_type_package(package, server_type)

        def get_ips_type(self, server_type):
            return self.master.get_server_ips_by_type(server_type) 
       
        def portage_install_type_package(self, package, server_type):
            return self.master.portage_install_type_package_threaded(package, server_type)

        def portage_pretend_type_world(self, server_type):
            return self.master.portage_pretend_type_world_threaded(package, server_type)
        
        def portage_pretend_type_package(self, package, server_type):
            return self.master.portage_pretend_type_package_threaded(package, server_type)

        def portage_sync_type(self, server_type):
            return self.master.portage_sync_type(server_type)

        def portage_search_type_package(self, package, server_type):
            return self.master.portage_search_type_package_threaded(package, server_type)

        def portage_pretend_type_package_process(self, package, server_type):
            return self.master.portage_pretend_type_package_process(package, server_type)
        
        def portage_latest_version_type_package(self, package, server_type):
            server_ips_package = self.master.get_server_ips_by_type(server_type)
            server_ips_package.append(package)
            logger.info(server_ips_package)
            return self.master.portage_latest_version_threaded(server_ips_package)

        def portage_latest_version_type_package_process(self, package, server_type):
            server_ips_package = self.master.get_server_ips_by_type(server_type)
            server_ips_package.append(package)
            logger.info(server_ips_package)
            return self.master.portage_latest_version_process(server_ips_package)

        def portage_ebuild_package_process(self, package, server_type):
            pass
            # server_ips_package = self.master.get_server_ips_by_type(server_type)
#             server_ips_package.append(package)
#             logger.info(server_ips_package)
#             return self.master.portage_ebuild_package_process(server_ips_package)

        def portage_ebuild_package(self, package):
            pass
        # return self.master.portage_ebuild_package_threaded(package)

        def portage_update_type_system(self, server_type):
            return self.master.portage_update_type_system_threaded(server_type)
        
        def portage_update_type_system_process(self, server_type):
            server_ips = self.master.get_server_ips_by_type(server_type)
            logger.info(server_ips)
            return self.master.portage_update_type_system_process(server_ips)

        def db_add_type(self, server_type):
            return self.master.add_type(server_type)

        def db_get_types(self):
            return self.master.get_types()

        def portage_push_files_type(self, server_type):
            return self.master.portage_push_files_type_threaded(server_type)

        def portage_push_file_type(self, file, server_type):
            return self.master.portage_push_files_type_threaded(server_type)


class PortageService(object):
    """
    Class to create an XMLRPCServer on given ip and port
    and export functions in given object
    """

    def __init__(self, ip, port, export_functions):
        self.ip = ip
        self.port = port
        self.export_functions = export_functions
        class RequestHandler(SimpleXMLRPCRequestHandler):
            rpc_paths = ('/RPC2',)

        self.server = SimpleXMLRPCServer((self.ip, self.port),
                                         requestHandler=RequestHandler, allow_none = True)
        self.server.register_multicall_functions()
        self.server.register_introspection_functions()
        self.server.register_instance(self.export_functions)
        self.service = kthread.KThread(target = self.serve, args = [])
        self.service.daemon = False
        self.service.start()

        
    def serve(self):
        try:
            self.server.serve_forever()
        except KeyboardInterrupt:
            logger.info("Got ctrl-c, exiting..")
            sys.exit(0)


def main():
    (options, args) = parser.parse_args()
    PortageMaster(options.ip,
                  options.port)




def test():
    # pserver = PortageServer('192.168.0.101', 'prod')
    # print pserver.portage_pretend('vim')
    # multiprocessing.freeze_support()
    pmaster = PortageMaster("192.168.1.200", _master_port)
    print pmaster.service_stop()
    # print pmaster.get_server_id("192.168.0.101")
    # pmaster.init_server_connections()
    # print pmaster.get_type_id('prod')
    # pmaster.add_server('portage-003', '192.168.0.102', 'prod')
    # print pmaster.get_server_ips_by_type('prod')
    # print pmaster.get_packages_by_type('qa')
    # print pmaster.portage_pretend_type_package_threaded('gcc', 'prod')
    # print pmaster.portage_pretend_type_package_process('gcc', 'prod')
    # print pmaster.portage_pretend_type_package_threaded('gcc', 'prod')
    # print pmaster.get_types()
    # print HelperDefs._get_bin_path()
    # print pmaster._make_type_dirs()
    # print pmaster.add_type('master')
    # print pmaster.get_server_ips()
    # print pmaster.get_server_ips_by_type('master')
#     print pmaster.get_type(pmaster.get_type_id_by_server('192.168.0.101'))

#     print pmaster.get_server_ips_by_type('master')
    # print pmaster.push_make_conf('192.168.0.101')
    # type = self.get_type(self.get_type_by_server(ip))


    # print pmaster.db_get_last_seen(9, datetime.datetime.now())
    # print pmaster.db_increment_failure(9, 11)
    # print pmaster.get_type(pmaster.get_type_id_by_server('192.168.1.104'))
    # print pmaster.portage_push_files_ip('192.168.0.101', 'prod')
    # print pmaster.get_server_ip_by_id(9)


if __name__ == "__main__":
    main()
    # test()
        
